package main

import (
	"fmt"
	 "github.com/PuerkitoBio/goquery"
    _ "strings"
	"github.com/gocolly/colly"
     "github.com/gocolly/colly/extensions"
     "regexp"
    "time"
)

func main(){
	t := time.Now()
	number := 1
	c := colly.NewCollector(func(c *colly.Collector){		//创建收集器
		extensions.RandomUserAgent(c)
		c.Async = true
	},
		colly.URLFilters(
			//过滤url，去除不是https://movie.douban.com/top250?start=0&filter= 的url
			regexp.MustCompile("^(https://movie\\.douban\\.com/top250)\\?start=[0-9].*&filter="),
		),
	)  
	// c := colly.NewCollector()

	//请求信息
	c.OnRequest(func(r *colly.Request){
		r.Headers.Set("Host", "movie.douban.com")
        r.Headers.Set("Connection", "keep-alive")
        r.Headers.Set("Accept", "*/*")
        r.Headers.Set("Origin", "https://movie.douban.com")
        r.Headers.Set("Referer", "https://movie.douban.com/top250?start=0&filter=") //关键头 如果没有 则返回 错误
        r.Headers.Set("Accept-Encoding", "gzip, deflate")
        r.Headers.Set("Accept-Language", "zh-CN,zh;q=0.9")

        fmt.Printf("%v \n",r)
	})

	// c.OnResponse(func(r *colly.Response){
	// 	fmt.Println("12")
	// })


	//响应的格式在html提取页面中的链接
	c.OnHTML("a[href]",func(e *colly.HTMLElement){
		link := e.Attr("href")
		c.Visit(e.Request.AbsoluteURL(link))
	})
	//获取电影信息
	c.OnHTML("div.info",func(e *colly.HTMLElement){
		e.DOM.Each(func(i int,selection *goquery.Selection){
			// movies := selection.Find("span.title").First().Text()
			// director := strings.Join(strings.Fields(selection.Find("div.bd p").First().Text())," ")
			// quote := selection.Find("p.quote span.inq").Text()
			// fmt.Printf("%d --> %s:%s  %s\n",number,movies,director,quote)
			number ++
		})
	})
	c.OnError(func(response *colly.Response,err error){
		fmt.Println(err)
	})
	c.Visit("https://movie.douban.com/top250?start=0&filter=")
	c.Wait()
	fmt.Printf("花费时间：%s",time.Since(t))

}